


### 色情 第一批数据

# 1：敏感语料 文德提供的短文本，词库，后期提供的较长文本，精加工标注的弹幕 8324 条

# 2：非敏感语料：文德提供的弹幕标注“非低俗”语料14941，精加工标注弹幕中“非敏感” 3732条

# 测试集比例0.1 2699 24298


# ## 统计
# import os
# count = 0

# path = "/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/文德语料_黄_挑选/sq"

# for root,dirs,files in os.walk(path):
#     for file in files:
#         if '.txt' in file:
#             file_name = os.path.join(root,file)
#             print(file_name)
#             with open(file_name,'r',encoding='utf-8') as f:
#                 for line in f.readlines():
#                     line = line.strip()
#                     if line:
#                         count += 1
# print(count)

# exit()


import pandas as pd
import os
import re
import random
random.seed(7)

data = pd.DataFrame(columns=['label','text'])

####  非敏感语料 ####
## 第一批弹幕测试文本中“非低俗”部分
df = pd.read_excel('/Users/leo/OneDrive/Work/项目工作/文德数慧-文本内容审核/分类实验/测试集结果.xlsx')
df =  df[df['备注1'] != '低俗'].reset_index(drop=True)

assert len(df) == 14941

for i in range(len(df)):
    label = 0
    text = df.iloc[i]['弹幕审核测试文案']
    text = text.replace('跟大家击掌加油✋','').strip()
    data = data.append(pd.DataFrame({'label':[label],'text':[text]}),ignore_index=True)

## 第一批精加工标注弹幕中“非敏感” 3732条
df = pd.read_excel('/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/result_select_5000(已标).xlsx')

df = df.fillna(0)

df = df[(df['非敏感'] == 1)].reset_index(drop=True)

assert len(df) == 3732

for i in range(len(df)):
    label = 0
    text = df.iloc[i]['弹幕内容']
    data = data.append(pd.DataFrame({'label':[label],'text':[text]}),ignore_index=True)



####  敏感语料 ####
path = "/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/文德语料_黄_挑选/sq"

for root,dirs,files in os.walk(path):
    for file in files:
        if '.txt' in file:
            file_name = os.path.join(root,file)
            print(file_name)
            with open(file_name,'r',encoding='utf-8') as f:
                for line in f.readlines():
                    line = line.strip()
                    if line:
                        label = 1
                        text = line
                        data = data.append(pd.DataFrame({'label':[label],'text':[text]}),ignore_index=True)



data['label'] = data['label'].astype(int)
data = data.sample(frac=1,random_state=777).reset_index(drop=True)

data_test = data[:int(len(data)*0.1)]
data_train = data[int(len(data)*0.1):]
print(len(data_test))
print(len(data_train))



data_train.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/sq/v1/train.tsv",sep='\t',header=False,index=False)
data_test.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/sq/v1/test.tsv",sep='\t',header=False,index=False)
data_test.to_csv(r"/Users/leo/Data/项目数据/文德数慧-文本内容审核/分类实验/数据/data_bert/sq/v1/dev.tsv",sep='\t',header=False,index=False)


